# -*- coding: utf-8 -*-
# time: 2025/4/12 10:17
# file: local_data_deal.py
# author: hanson

"""
本地数据处理

"""
from datasets import load_dataset
from modelscope import MsDataset

data_id=r"F:\temp\pediatrics-dataset\train.csv"
dataset = MsDataset.load(
    data_id,
    subset_name='default',
    split='train',
    cache_dir="./cache" # 指定缓存目录
).to_hf_dataset()
print(dataset)  # 输出字段结构及示例数据

dataset = load_dataset("csv",data_files=data_id, split="train",trust_remote_code=True)
print(dataset)  # 输出字段结构及示例数据

dataset = dataset.select(range(10))

# 动态字段适配
def detect_columns(df):
    column_map = {
        'instruction': ['Instruction', 'title', '任务'],
        'input': ['input', 'content', '问题'],
        'output': ['output', 'answer', '回复']
    }
    detected = {}
    for target, candidates in column_map.items():
        for col in df:
            if col in candidates:
                detected[target] = col
                break
    return detected

#执行字段转换

def convert_to_llm_format(df):
    detected_cols = detect_columns(df)

    # 动态生成训练样本‌:ml-citation{ref="1,7" data="citationList"}
    formatted_data = []
    for _, row in df:
        instruction = row.get(detected_cols.get('instruction', ''), "")
        user_input = row.get(detected_cols.get('input', ""), "")
        assistant_output = row.get(detected_cols.get('output', ""), "")

        formatted_data.append({
            "messages": [
                {"role": "user", "content": f"{instruction}\n{user_input}".strip()},
                {"role": "assistant", "content": assistant_output}
            ]
        })
    return formatted_data

print(convert_to_llm_format(dataset))